{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import re\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_string(string):\n",
    "    string = re.sub('[^A-Za-z0-9\\-\\/ ]+', ' ', string).split()\n",
    "    return [y.strip() for y in string]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('pos-data-v3.json','r') as fopen:\n",
    "    dataset = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['뭘봐', '뭘봐', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['ひ', 'ひ', 'PROPN']\n",
      "list index out of range ['ヒ', 'ヒ', 'PROPN']\n",
      "list index out of range ['形聲', '形聲', 'NOUN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['汉', '汉', 'PROPN']\n",
      "list index out of range ['东', '东', 'PROPN']\n",
      "list index out of range ['王', '王', 'PROPN']\n",
      "list index out of range ['（', '（', 'PROPN']\n",
      "list index out of range ['伊', '伊', 'PROPN']\n",
      "list index out of range ['）', '）', 'PROPN']\n",
      "list index out of range ['ȝ', 'ȝ', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range [\"'\", '_', 'PROPN']\n",
      "list index out of range ['碁', '碁', 'NOUN']\n",
      "list index out of range ['囲碁', '囲碁', 'NOUN']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['会', '会', 'PROPN']\n",
      "list index out of range ['蔡武侯', '蔡武侯', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['*', '*', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['~', '~', 'ADJ']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['ОАО', 'оао', 'PROPN']\n",
      "list index out of range ['Газпром', 'газпром', 'PROPN']\n",
      "list index out of range [\"''\", '_', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['佐々木', '佐々木', 'PROPN']\n",
      "list index out of range ['功', '功', 'PROPN']\n",
      "list index out of range ['ɳ', 'ɳ', 'PROPN']\n",
      "list index out of range ['ʂ', 'ʂ', 'PROPN']\n",
      "list index out of range ['ʈ', 'ʈ', 'PROPN']\n",
      "list index out of range ['ɖ', 'ɖ', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range [\"'\", '_', 'NOUN']\n",
      "list index out of range [\"'\", '_', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['鉄の森', '鉄の森', 'PROPN']\n",
      "list index out of range ['アイゼンヴァルト', 'アイゼンヴァルト', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['壽', '壽', 'PROPN']\n",
      "list index out of range ['陽', '陽', 'PROPN']\n",
      "list index out of range ['裴叔業', '裴叔業', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['فيروز', 'فيروز', 'PROPN']\n",
      "list index out of range ['الديلامي', 'الديلامي', 'PROPN']\n",
      "list index out of range ['נביאים', 'נביאים', 'NOUN']\n",
      "list index out of range ['ראשונים', 'ראשונים', 'NOUN']\n",
      "list index out of range ['נביאים', 'נביאים', 'NOUN']\n",
      "list index out of range ['جامع', 'جامع', 'PROPN']\n",
      "list index out of range ['الرئيس', 'الرئيس', 'PROPN']\n",
      "list index out of range ['الصالح', 'الصالح', 'PROPN']\n",
      "list index out of range ['—', '—', 'NOUN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['ä', 'ä', 'PROPN']\n",
      "list index out of range ['ʈ', 'ʈ', 'PROPN']\n",
      "list index out of range ['ʈʰ', 'ʈʰ', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range [\"''\", '_', 'PROPN']\n",
      "list index out of range ['€', '€', 'SYM']\n",
      "list index out of range ['\\u200e', '\\u200e', 'VERB']\n",
      "list index out of range ['\\u200e', '\\u200e', 'NOUN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['$', '$', 'SYM']\n",
      "list index out of range ['Α', 'α', 'PROPN']\n",
      "list index out of range ['А', 'а', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['$', '$', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['おる', 'おる', 'PROPN']\n",
      "list index out of range ['いる', 'いる', 'PROPN']\n",
      "list index out of range ['じゃ', 'じゃ', 'PROPN']\n",
      "list index out of range ['や', 'や', 'PROPN']\n",
      "list index out of range ['だ', 'だ', 'PROPN']\n",
      "list index out of range ['～', '～', 'PROPN']\n",
      "list index out of range ['ん', 'ん', 'PROPN']\n",
      "list index out of range ['わから', 'わから', 'PROPN']\n",
      "list index out of range ['ん', 'ん', 'PROPN']\n",
      "list index out of range ['～', '～', 'PROPN']\n",
      "list index out of range ['ない', 'ない', 'PROPN']\n",
      "list index out of range ['わからない', 'わからない', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['×', '×', 'NOUN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['–', '–', 'NUM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['続', '続', 'PROPN']\n",
      "list index out of range ['・', '・', 'PROPN']\n",
      "list index out of range ['三', '三', 'PROPN']\n",
      "list index out of range ['丁目', '丁目', 'PROPN']\n",
      "list index out of range ['の', 'の', 'PROPN']\n",
      "list index out of range ['夕日', '夕日', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['Ἑλλάδος', 'ἑλλάδος', 'PROPN']\n",
      "list index out of range ['παίδευσις', 'παίδευσις', 'PROPN']\n",
      "list index out of range ['€', '€', 'SYM']\n",
      "list index out of range ['ê', 'ê', 'PROPN']\n",
      "list index out of range ['é', 'é', 'PROPN']\n",
      "list index out of range ['è', 'è', 'PROPN']\n",
      "list index out of range ['ë', 'ë', 'PROPN']\n",
      "list index out of range ['ē', 'ē', 'PROPN']\n",
      "list index out of range ['ĕ', 'ĕ', 'PROPN']\n",
      "list index out of range ['ě', 'ě', 'PROPN']\n",
      "list index out of range ['ẽ', 'ẽ', 'PROPN']\n",
      "list index out of range ['ė', 'ė', 'PROPN']\n",
      "list index out of range ['ę', 'ę', 'PROPN']\n",
      "list index out of range ['ẻ', 'ẻ', 'PROPN']\n",
      "list index out of range ['ö', 'ö', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['.', '.', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['بندر', 'بندر', 'PROPN']\n",
      "list index out of range ['عباس', 'عباس', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['+', '+', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['$', '$', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['珠海', '珠海', 'PROPN']\n",
      "list index out of range [\"''\", '_', 'NOUN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['서종', '서종', 'PROPN']\n",
      "list index out of range ['제', '제', 'PROPN']\n",
      "list index out of range ['±', '±', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['仙', '仙', 'PROPN']\n",
      "list index out of range ['仚', '仚', 'PROPN']\n",
      "list index out of range ['僊', '僊', 'PROPN']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['腹切り', '腹切り', 'PROPN']\n",
      "list index out of range ['§', '§', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['육식', '육식', 'PROPN']\n",
      "list index out of range ['동물', '동물', 'PROPN']\n",
      "list index out of range ['도망자', '도망자', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['الغرب', 'الغرب', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['سدوم', 'سدوم', 'PROPN']\n",
      "list index out of range ['عمورة', 'عمورة', 'PROPN']\n",
      "list index out of range ['高雄', '高雄', 'PROPN']\n",
      "list index out of range ['大眾', '大眾', 'PROPN']\n",
      "list index out of range ['捷', '捷', 'PROPN']\n",
      "list index out of range ['運', '運', 'PROPN']\n",
      "list index out of range ['系統', '系統', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['=', '=', 'SYM']\n",
      "list index out of range ['亳', '亳', 'PROPN']\n",
      "list index out of range ['°', '°', 'SYM']\n",
      "list index out of range ['المعقبات', 'المعقبات', 'PROPN']\n",
      "list index out of range ['%', '%', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['#', '#', 'SYM']\n",
      "list index out of range ['%', '%', 'SYM']\n"
     ]
    }
   ],
   "source": [
    "texts, labels = [], []\n",
    "for i in dataset:\n",
    "    try:\n",
    "        texts.append(process_string(i[0])[0].lower())\n",
    "        labels.append(i[-1])\n",
    "    except Exception as e:\n",
    "        print(e, i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(103417, 28374)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = LabelEncoder().fit_transform(labels)\n",
    "bow_chars = CountVectorizer(ngram_range=(2, 4), analyzer='char').fit(texts)\n",
    "vectors = bow_chars.transform(texts)\n",
    "vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(vectors, target, test_size = 0.1)\n",
    "del vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5,  0, 10, 13,  6,  0,  2, 10,  6,  6])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_Y[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ADJ       0.64      0.58      0.61      4101\n",
      "        ADP       0.88      0.89      0.88     10755\n",
      "        ADV       0.68      0.75      0.71      4285\n",
      "        AUX       0.96      1.00      0.98       970\n",
      "      CCONJ       0.95      0.92      0.94      3268\n",
      "        DET       0.85      0.92      0.88      3630\n",
      "       NOUN       0.70      0.76      0.73     24297\n",
      "        NUM       0.93      0.74      0.82      3936\n",
      "       PART       0.92      0.70      0.80       532\n",
      "       PRON       0.96      0.80      0.87      4296\n",
      "      PROPN       0.78      0.70      0.74     20432\n",
      "      SCONJ       0.65      0.66      0.66      1327\n",
      "        SYM       0.00      0.00      0.00       222\n",
      "       VERB       0.74      0.86      0.80     10989\n",
      "          X       1.00      0.11      0.21        35\n",
      "\n",
      "avg / total       0.78      0.78      0.77     93075\n",
      "\n"
     ]
    }
   ],
   "source": [
    "multinomial = MultinomialNB().fit(train_X, train_Y)\n",
    "print(metrics.classification_report(train_Y, multinomial.predict(train_X), target_names = np.unique(labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('multinomial-pos.pkl','wb') as fopen:\n",
    "    pickle.dump(multinomial,fopen)\n",
    "with open('bow-pos.pkl','wb') as fopen:\n",
    "    pickle.dump(bow_chars,fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
